﻿using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;

namespace ConsoleApp9
{

    internal class Program
    {
        // 控制并发数量的信号量
        private static readonly SemaphoreSlim _semaphore = new SemaphoreSlim(5); // 同时最多5个请求
        private static readonly HttpClient _httpClient = new HttpClient();
        private static readonly ConcurrentDictionary<string, HashSet<string>> _phoneNumbers = new ConcurrentDictionary<string, HashSet<string>>();
        private static readonly ConcurrentBag<string> _processedUrls = new ConcurrentBag<string>();
        private static CancellationTokenSource _cancellationTokenSource;

        // 电话号码正则表达式（匹配中国大陆电话号码）
        private static readonly Regex _phoneRegex = new Regex(@"(?:\+?86)?[-\s]?(?:\b1[3-9][0-9]{9}\b|\(?\d{3,4}\)?[-\s]?\d{7,8})");
        static async Task Main(string[] args)
        {
            Console.WriteLine("请输入搜索关键字（如：大学 联系电话）：");
            var keyword = Console.ReadLine();

            if (string.IsNullOrWhiteSpace(keyword))
            {
                Console.WriteLine("搜索关键字不能为空");
                return;
            }

            // 初始化HTTP客户端
            _httpClient.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64)");
            _httpClient.Timeout = TimeSpan.FromSeconds(10);

            // 重置状态
            _phoneNumbers.Clear();
            _cancellationTokenSource = new CancellationTokenSource();

            Console.WriteLine("开始爬取... (按任意键取消)");

            // 启动取消监听的线程
            var cancelTask = Task.Run(() =>
            {
                Console.ReadKey(true);
                _cancellationTokenSource.Cancel();
                Console.WriteLine("\n正在取消爬取...");
            });

            try
            {
                var crawlingTask = StartCrawling(keyword, _cancellationTokenSource.Token);
                await Task.WhenAny(crawlingTask, cancelTask);

                if (crawlingTask.IsCompleted)
                {
                    Console.WriteLine($"\n爬取完成，共找到{_phoneNumbers.Sum(p => p.Value.Count)}个电话号码");
                    PrintResults();
                }
            }
            catch (OperationCanceledException)
            {
                Console.WriteLine("\n爬取已取消");
            }
            catch (Exception ex)
            {
                Console.WriteLine($"\n发生错误: {ex.Message}");
            }
            Console.ReadKey(true);
        }
        static async Task StartCrawling(string keyword, CancellationToken cancellationToken)
        {
            // 1. 使用搜索引擎获取初始URL列表
            var searchUrls = await GetSearchResults(keyword, cancellationToken);

            // 2. 并行处理搜索结果
            var tasks = searchUrls.Select(url => ProcessPageAsync(url, cancellationToken));
            await Task.WhenAll(tasks);

            // 3. 如果电话号码不足50个，继续从已找到的页面中挖掘更多链接
            while (_phoneNumbers.Sum(p => p.Value.Count) < 50 && !cancellationToken.IsCancellationRequested)
            {
                var newUrls = _processedUrls
                    .SelectMany(url => ExtractLinksFromHtml(GetPageHtml(url).Result))
                    .Where(url => !_processedUrls.Contains(url))
                    .Take(20) // 每次最多处理20个新URL
                    .ToList();

                if (!newUrls.Any()) break;

                var newTasks = newUrls.Select(url => ProcessPageAsync(url, cancellationToken));
                await Task.WhenAll(newTasks);
            }
        }

        static async Task<List<string>> GetSearchResults(string keyword, CancellationToken cancellationToken)
        {
          
            await Task.Delay(500, cancellationToken); // 模拟网络延迟

            return new List<string>
        {
            "https://www.pku.edu.cn",
            "https://www.zju.edu.cn",
            "https://www.nju.edu.cn",
            "https://www.ustc.edu.cn",
            "https://www.ruc.edu.cn",
            "https://www.bnu.edu.cn",
        };
        }

        static async Task ProcessPageAsync(string url, CancellationToken cancellationToken)
        {
            await _semaphore.WaitAsync(cancellationToken);
            try
            {
                if (_processedUrls.Contains(url) || cancellationToken.IsCancellationRequested)
                    return;

                Console.WriteLine($"正在处理: {url}");

                string html;
                try
                {
                    html = await GetPageHtml(url);
                    _processedUrls.Add(url);
                }
                catch (Exception ex)
                {
                    Console.WriteLine($"获取页面失败: {url}, 错误: {ex.Message}");
                    return;
                }

                // 提取电话号码
                var matches = _phoneRegex.Matches(html);
                foreach (Match match in matches)
                {
                    var phone = NormalizePhoneNumber(match.Value);
                    _phoneNumbers.AddOrUpdate(phone,
                        new HashSet<string> { url },
                        (_, set) => { set.Add(url); return set; });
                }

                // 如果已经找到足够多的电话号码，提前结束
                if (_phoneNumbers.Count >= 50)
                {
                    _cancellationTokenSource.Cancel();
                }
            }
            finally
            {
                _semaphore.Release();
            }
        }

        static async Task<string> GetPageHtml(string url)
        {
            try
            {
                var response = await _httpClient.GetAsync(url);
                response.EnsureSuccessStatusCode();
                return await response.Content.ReadAsStringAsync();
            }
            catch
            {
                // 失败时重试一次
                await Task.Delay(1000);
                var response = await _httpClient.GetAsync(url);
                response.EnsureSuccessStatusCode();
                return await response.Content.ReadAsStringAsync();
            }
        }

        static List<string> ExtractLinksFromHtml(string html)
        {
            if (string.IsNullOrWhiteSpace(html))
                return new List<string>();

            try
            {
                var linkRegex = new Regex(
                    @"<a\b[^>]*?\bhref\s*=\s*[""']?(?<url>[^""'\s>]+)[""']?[^>]*>",
                    RegexOptions.IgnoreCase | RegexOptions.Compiled);

                var links = new List<string>();

                foreach (Match match in linkRegex.Matches(html))
                {
                    if (match.Success)
                    {
                        string url = match.Groups["url"].Value;

                        // 处理相对URL
                        if (!url.StartsWith("http") && !url.StartsWith("//"))
                            continue;

                        // 规范化URL
                        if (url.StartsWith("//"))
                            url = "https:" + url;

                        // 添加到结果列表
                        links.Add(url);
                    }
                }

                return links.Distinct().ToList();
            }
            catch (Exception ex)
            {
                Console.WriteLine($"提取链接时出错: {ex.Message}");
                return new List<string>();
            }
        }

        static string NormalizePhoneNumber(string phone)
        {
            // 标准化电话号码格式
            return new string(phone.Where(char.IsDigit).ToArray());
        }

        static void PrintResults()
        {
            Console.WriteLine("\n爬取结果：");
            Console.WriteLine($"已处理URL数量: {_processedUrls.Count}");
            Console.WriteLine($"找到的唯一电话号码数量: {_phoneNumbers.Count}");

            Console.WriteLine("\n电话号码及来源URL:");
            foreach (var pair in _phoneNumbers.OrderBy(p => p.Key))
            {
                Console.WriteLine($"\n电话: {pair.Key}");
                foreach (var url in pair.Value)
                {
                    Console.WriteLine($"  - {url}");
                }
            }

            Console.WriteLine("\n已爬取的URL列表:");
            foreach (var url in _processedUrls.OrderBy(u => u))
            {
                Console.WriteLine($"- {url}");
            }
        }
    }
}
